import scrapy
from styletable.items import StyletableItem
import re


class GoodsSpider(scrapy.Spider):
    name = 'goods'
    allowed_domains = ['bkc.styletable.jp']
    start_urls = ['https://bkc.styletable.jp/']

    def parse(self, response):
        for title in response.css('li.large_item,li.medium_item'):
            self.logger.info(title.css('a::text').get())
            self.logger.info(title.css('a').attrib['href'])
            list_url = title.css('a').attrib['href']
            yield scrapy.Request(response.urljoin(list_url), self.parse_goods_list, meta={'category_name' : title.css('a::text').get()})

    def parse_goods_list(self, response):
        if response.css('li.ec-pager__item:last-child a::text').get() == '次へ':
            next_page = response.css('li.ec-pager__item:last-child a').attrib['href']
            self.logger.info(next_page)
            yield scrapy.Request(response.urljoin(next_page), self.parse_goods_list,meta=response.meta)

        for goods in response.css('li.ec-shelfGrid__item'):
            details_url = goods.css('a').attrib['href']
            yield scrapy.Request(details_url, self.parse_goods_details,meta=response.meta,dont_filter=True)

    def parse_goods_details(self, response):
        product_name = response.css('.ec-headingTitle::text').get()
        self.logger.info('产品名称:' + product_name)
        product_price = response.css('.ec-price__price::text').get().replace(',', '').replace('円', '')
        self.logger.info('价格:' + product_price)
        product_images = response.css('.ec-sliderItemRole .slide-item img::attr(src)').getall()
        content = ''
        for desc in response.css('.ec-productRole__description > .tab-area'):
            content += '<p>' + desc.css('.tabs-title::text').get() + '</p>\n'
            content += desc.css('.tabs-detail').get()
        content = re.sub('<style>[\\w\\W]*</html>', '', content)
        item = StyletableItem()
        item['name'] = product_name
        item['category_name'] = response.meta['category_name']
        item['raw_id'] = response.url.split('/')[-1]

        def imgs_handler(url):
            if url[0] == '/':
                return 'https://bkc.styletable.jp/'+url
            return url

        product_images = list(map(imgs_handler, product_images))
        item['image_urls'] = product_images
        item['thumbs'] = product_images[0]
        item['pictures'] = ','.join(product_images)
        item['content'] = self.filter_tags(content)
        item['specs'] = '[{"spec_id":1,"spec_name":"%s","spec_num":"10000","price":%s,"thumb":"%s"}]' % (
            product_name, product_price, product_images[0])

        if not item['name'].startswith('【定期'):
            yield item

    def filter_tags(self, html):

        # 先过滤CDATA
        re_cdata = re.compile('//<!\[CDATA\[[^>]*//\]\]>', re.I)  # 匹配CDATA
        re_script = re.compile('<\s*script[^>]*>[^<]*<\s*/\s*script\s*>', re.I)  # Script
        re_style = re.compile('<\s*style[^>]*>[^<]*<\s*/\s*style\s*>', re.I)  # style
        re_br = re.compile('<br\s*?/?>')  # 处理换行
        re_h = re.compile('</?\w+[^>]*>')  # HTML标签
        re_comment = re.compile('<!--[^>]*-->')  # HTML注释
        s = re_cdata.sub('', html)  # 去掉CDATA
        s = re_script.sub('', s)  # 去掉SCRIPT
        s = re_style.sub('', s)  # 去掉style
        s = re_br.sub('\n', s)  # 将br转换为换行
        s = re_h.sub('', s)  # 去掉HTML 标签
        s = re_comment.sub('', s)  # 去掉HTML注释
        s = re.sub('\\s{2,}', '\n', s)
        # 去掉多余的空行
        blank_line = re.compile('\n+')
        s = blank_line.sub('\n', s)
        s = self.replaceCharEntity(s)  # 替换实体
        return s

    ##替换常用HTML字符实体.
    # 使用正常的字符替换HTML中特殊的字符实体.
    # 你可以添加新的实体字符到CHAR_ENTITIES中,处理更多HTML字符实体.
    # @param htmlstr HTML字符串.
    def replaceCharEntity(self, htmlstr):
        CHAR_ENTITIES = {'nbsp': ' ', '160': ' ',
                         'lt': '<', '60': '<',
                         'gt': '>', '62': '>',
                         'amp': '&', '38': '&',
                         'quot': '"', '34': '"', }

        re_charEntity = re.compile(r'&#?(?P<name>\w+);')
        sz = re_charEntity.search(htmlstr)
        while sz:
            entity = sz.group()  # entity全称，如>
            key = sz.group('name')  # 去除&;后entity,如>为gt
            try:
                htmlstr = re_charEntity.sub(CHAR_ENTITIES[key], htmlstr, 1)
                sz = re_charEntity.search(htmlstr)
            except KeyError:
                # 以空串代替
                htmlstr = re_charEntity.sub('', htmlstr, 1)
                sz = re_charEntity.search(htmlstr)
        return htmlstr
